#include "aceMesh_clang.h"

#include <cassert>
#include <cstdarg>
#include <vector>

#include <math.h>
#include "aceMesh_scheduler_init.h"
#include "aceMesh_task.h"
#include "aceMesh_utils.h"
#include "aceMesh_composite_task.h"
#include "utils/acelog.h"
#include "hashtable.h"
//#ifdef MEMORY_POOL
#include "MemPool.h"
//#endif

#ifdef THREAD_TRACE
extern "C"{
#include "thread_trace.h"
}
unsigned long begin_time;
unsigned long diff;
unsigned long trace_time;
FILE *f;
int trace_num_threads;
unsigned long traces_time[65][MAXB/4];
char traces_id[65][MAXB];
int total_m=0;
int total[64]={0};
#endif

#ifdef TASK_LIFETIME_PROFILING
unsigned long avr_generate_cycles = 0;
unsigned long avr_execute_cycles[64] = {0};
unsigned long avr_process_cycles[64] = {0};
#endif

#ifdef MAIN_MUTEX_PROFILING
extern unsigned int fail_count;
extern unsigned int success_count;
#endif

using namespace AceMesh_runtime;

#ifdef NO_PARALLEL
#include "task_dag_graph.h"
typedef std::vector<addr_tuple> VectorType;
#else
#include "concurrent_task_dag_graph.h"
typedef concurrent_task_dag_graph task_dag_graph;
typedef tbb::enumerable_thread_specific<std::vector<addr_tuple> > VectorType;
#endif
std::tr1::unordered_map<void *, DataAccess> local_addr;

#ifdef ACEMESH_GRAPH_BUILD_PROFILING
unsigned long buil_prof[N_BUILD_PROF];
#ifdef FINE_PROF2
unsigned long find_hash_ntimes1=0,find_hash_ntimes2=0;
#endif
#endif

#ifdef REUSE_GRAPH
int graph_reuse_times;
#endif

#ifdef ACEMESH_SCHEDULER_PROFILING
unsigned long total_nedges;
unsigned long total_ntasks_register;
#endif

#ifdef EMP_STEALING_PROF
unsigned emp_stealing_fail[64] = {0};
unsigned emp_stealing_success[64] = {0};
#endif

#ifdef THREADS_IDLING_PROF
unsigned thread_get_count[64] = {0};
unsigned thread_idle_count[64] = {0};
#endif


#ifdef MEMORY_POOL
extern struct MemPool pool;
extern "C" void InitPool();
extern "C" void slave_initpool();
extern "C" void *acemesh_myalloc_aligned_4(int datasize);
extern "C" void *acemesh_myalloc_aligned_8(int datasize);
extern "C" void *acemesh_myalloc_aligned_16(int datasize);
extern "C" void *acemesh_myalloc_aligned_32(int datasize);
extern "C" void ReInitial();
#endif
extern "C" void *my_malloc_aligned(int size);

VectorType _v_addr_tuple;
int _sepid = 0;
int loop_id = 0;
#ifdef TEMP_NESTED
#define NESTED_STACKSIZE 10
extern task_dag_graph task_graphs[NESTED_STACKSIZE];
extern int cur_nested_lev;
#else
extern task_dag_graph task_graph;
#endif
void *cur_super_taskptr;
//#ifdef DEBUG
extern int my_mpi_rank;
//#endif
int tag = 0;
int is_master = 0;

#ifdef LDMM
struct layout_info tilelayout[MAX_DTILEBOOK];
// consistency maintenance table
struct access_arrays *invalidateTab = NULL;
#endif

#ifdef OLD_STYLE
extern "C" void *get_taskfunc_pointer(int funcno);
#endif
inline void set_task_type(struct aceMesh_task *self, task_type type) { self->my_type = type; }

#ifdef LDMM
void set_paraLoops()
{
    int i;
    invalidateTab = (struct access_arrays *)malloc(sizeof(struct access_arrays) * TOTAL_LOOPS);
    for (i = 0; i < TOTAL_LOOPS; i++) {
        // invalidateTab[i]= (access_arrays*)malloc(sizeof(access_arrays));
        invalidateTab[i].num = 0;
        invalidateTab[i].arrList =
            (struct access_region *)malloc(sizeof(struct access_region) * ARRAY_NUMS);
    }
}

void clear_paraloops()
{
    int i;
    for (i = 0; i < TOTAL_LOOPS; i++) {
        free(invalidateTab[i].arrList);
    }
    free(invalidateTab);
    invalidateTab = NULL;
}

extern "C" void acemesh_register_invalidatetab(void *array_addr, int ndim, int *tile_no)
{
    int i, j, array_id;
    if (invalidateTab == NULL) set_paraLoops();
    array_id = invalidateTab[loop_id].num;
    invalidateTab[loop_id].arrList[array_id].array = array_addr;
    invalidateTab[loop_id].arrList[array_id].ndim = ndim;
    for (i = 0; i < ndim; ++i)
        for (j = 0; j < 2; ++j) {
            invalidateTab[loop_id].arrList[array_id].region[i][j] = *(tile_no + i * 2 + j);
            // printf("%d,",*(tile_no+i*2+j));
        }
    // printf("\n");
    for (i = ndim; i < MAX_DIM; i++)
        for (j = 0; j < 2; j++) invalidateTab[loop_id].arrList[array_id].region[i][j] = -1;
    invalidateTab[loop_id].num++;
}

extern "C" void ldmbuf_add(int *tilesize, int dim, int tile_num)  // add by gxr
{
    int i;
    for (i = 0; i < dim; i++) {
        tilelayout[tag].tile_size[i] = tilesize[i];
    }
    tilelayout[tag].Ntiles = tile_num;
    tag++;
}

extern "C" void acemesh_region_begin()
{
    tag = 0;
    // TODO
}
#endif

#ifdef THREAD_TRACE
extern "C" void acemesh_trace_print(const char event_name[]){
   
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',event_name);

}
#endif

#ifdef CONCURRENT_CONSTRUCT_GRAPH
extern "C" void acemesh_wait_for_all_task()
{
    wait_for_all_task();
}
#endif

#ifdef MTEST_LIGHT
#ifdef NO_PARALLEL
extern int *mtest_handle1;
extern int *mtest_handle2;
extern int mtest_kind1;
extern int mtest_kind2;
#else
extern tbb::enumerable_thread_specific<int *> mtest_handle1;
extern tbb::enumerable_thread_specific<int *> mtest_handle2;
extern tbb::enumerable_thread_specific<int> mtest_kind1;
extern tbb::enumerable_thread_specific<int> mtest_kind2;
#endif
#endif
#ifdef NO_PARALLEL
ci_task *cur_task;
#else
tbb::enumerable_thread_specific<ci_task *> cur_task;

#endif

// expose task interface to those using acemesh_task_generator()

extern "C" void acemesh_task_set_type(task_type type)
{
#ifdef NO_PARALLEL
    set_task_type(&(cur_task->aceMesh_task_base), type);
#else
    (ci_task *)&local_cur_task = &(cur_task.local());
    set_task_type(local_cur_task, type);
#endif
}

extern "C" void acemesh_task_set_affinity(int id)
{
#ifdef NO_PARALLEL
    set_affinity_id(&(cur_task->aceMesh_task_base.task_base), id);
#else
    (ci_task *)&local_cur_task = &(cur_task.local());
    set_affinity_id(local_cur_task, id);
#endif
}

extern int sche_num_threads;
int g_init_offset = 0;

int sub_affinity_startid = 0;
//#ifdef TEMP_NESTED
//int local_cur_nested=0;
//#endif
extern "C" void acemesh_task_set_sub_affinity(int width)
{
#ifdef AUTO_AFFINITY
#ifdef USE_COMPOSITE_TASK
//    printf("startid: %d, size: %d, ", sub_affinity_startid, width);
//    local_cur_nested=(++local_cur_nested)%NESTED_STACKSIZE;
//#ifdef TEMP_NESTED
//    printf("cur_nested_lev=%d",local_cur_nested);
//#endif
//    printf("\n");
//    fflush(stdout);
//#ifdef TEMP_NESTED
////    set_sub_affinity_id(&task_graphs[cur_nested_lev].my_affinity, sub_affinity_startid, width);
//    set_sub_affinity_id(&task_graphs[local_cur_nested].my_affinity, sub_affinity_startid, width);
//#else
//    set_sub_affinity_id(&((aceMesh_composite_task*)cur_task)->graph.my_affinity, sub_affinity_startid, width);
//#endif
    if(cur_task->aceMesh_task_base.my_type != COMPOSITE_TASK)
    {
        assert(0);
    }
    ((aceMesh_composite_task*)cur_task)->nested_width = width;
    ((aceMesh_composite_task*)cur_task)->nested_begin_index = sub_affinity_startid;
    sub_affinity_startid += width;
    if (sub_affinity_startid + width > sche_num_threads) {
        sub_affinity_startid = 0;
    }
#endif
#endif
}

extern "C" void acemesh_task_set_sub_affinity_(int width)
{
    acemesh_task_set_sub_affinity(width);
}



#if defined(USE_PRIORITY_QUEUE) || defined(LOCAL_MULTI_PRIORI)
extern "C" void acemesh_task_set_priority_id(int id)
{
    set_priority_id(&(cur_task->aceMesh_task_base.task_base), id);
}
extern "C" int acemesh_task_get_priority_id()
{
    return get_priority_id(&(cur_task->aceMesh_task_base.task_base));
}
#endif

#ifdef AUTO_AFFINITY
#ifdef TEMP_NESTED
extern "C" void acemesh_reset_affinity() { task_graphs[cur_nested_lev].reset_affinity_id(); }
#else
extern "C" void acemesh_reset_affinity() { task_graph.reset_affinity_id(); }
#endif
#endif

unsigned long rpcc()
{
    unsigned long time;
    asm("rtc %0" : "=&r"(time) ::"memory");
    return time;
}
#ifdef ACEMESH_PARA
extern int interval;
extern int gid1, gid2;
extern unsigned long time_init, tt_build;

extern "C" void acemesh_set_interval(int it) { interval = it; }

#endif

/*
#ifdef ACEMESH_SCHEDULER_PROFILING
unsigned long execute_start = 0;
unsigned long execute_end = 0;
extern int total_num_threads;
#endif

#ifdef ACEMESH_TIME
extern double all_pure_exec_time[65];
#endif
*/
#ifdef MASTER
void acemesh_task_map_master() { is_master = 1; }

extern "C" struct task *user_execute(ci_task *self)
{
#ifdef TEMP_ASSERT
    if(self->aceMesh_task_base.task_base.affinity_id != sche_num_threads) assert(0);
#endif
    if (self->aceMesh_task_base.my_type == COMPOSITE_END_TASK) {
//        printf("XXXXXXXcomposite end task=%p\n",self);
//        fflush(stdout);
        return execute((struct aceMesh_task *)self);
    }
  //  printf("begin_user_execute\n");
#ifdef ACEMESH_PARA
    long mcur_time;
#endif
    cur_task = self;
    alog_debug("rank: %d,before ci_task_func", my_mpi_rank);
    //printf("rank: %d,before ci_task_func,task=%p\n", my_mpi_rank,self);
    //fflush(stdout);
#ifdef ACEMESH_PARA
    mcur_time = rpcc();
    gid1 = (mcur_time - time_init - tt_build) / interval;
#endif

    //if composite pass current task ptr to aceMesh_clang.cpp
    if (self->aceMesh_task_base.my_type == COMPOSITE_TASK) {
        alog_debug("composite task, self addr: %x", self);
#ifdef TEMP_NESTED
        cur_super_taskptr = self;
        //cur_nested_lev=((++cur_nested_lev)%NESTED_STACKSIZE);
        cur_nested_lev++;
        //printf("XXXX++cur_nested_lev=%d\n",cur_nested_lev);
        int width = ((aceMesh_composite_task*)cur_task)->nested_width;
        int startid = ((aceMesh_composite_task*)cur_task)->nested_begin_index;
        set_sub_affinity_id(&task_graphs[cur_nested_lev].my_affinity,startid,width);
#else
        cur_super_taskptr = self;
#endif
    }

//#ifdef TEMP_NESTED
        alog_debug("task user_execute, self addr: %x", self);
        //printf("task user_execute, self addr: %p,%p,%d\n", self,self->slave_ci_task_func,self->aceMesh_task_base.task_base.affinity_id);

/*
#ifdef ACEMESH_SCHEDULER_PROFILING
        execute_start = rpcc();
#endif
*/

//#ifdef TEMP_MODI_MASTER
#ifdef DOUBLE_FUNC
        self->master_ci_task_func(self->ci_args);
#else
        self->slave_ci_task_func(self->ci_args);
#endif
//#endif

/*
#ifdef ACEMESH_SCHEDULER_PROFILING
        execute_end = rpcc();
        all_pure_exec_time[total_num_threads] += (double)(execute_end - execute_start)/MFREQ;
#endif
*/

        alog_debug("task user_execute after slave_func");
#ifdef DEBUG
        printf("task user_execute after slave_func,%p,%p,type=%d,%d\n",self,self->slave_ci_task_func,self->aceMesh_task_base.my_type,self->aceMesh_task_base.task_base.affinity_id);
        fflush(stdout);
#endif
//#else
//    if (self->aceMesh_task_base.my_type == COMPOSITE_END_TASK) {
//        //when cur_task is composite end task do not access funcptr
//    } else {
//        alog_debug("task user_execute, self addr: %x", self);
//        //printf("task user_execute, self addr: %p\n", self);
//#ifdef TEMP_MODI_MASTER
//        self->slave_ci_task_func(self->ci_args);
//#endif
//        alog_debug("task user_execute after slave_func");
//        //printf("task user_execute after slave_func");
//
//    }
//#endif

    //after composite execute reset build ctx
    if (self->aceMesh_task_base.my_type == COMPOSITE_TASK) {
        cur_super_taskptr = NULL;
#ifdef TEMP_NESTED
        cur_nested_lev--;
//        printf("XXXXcur_nested_lev=%d\n",cur_nested_lev);
#endif
#ifdef TEMP_DEBUG
        printf("I'm %p,composite's vertical:%p,vert_ref=%d\n",self,self->aceMesh_task_base.vertical_task,self->aceMesh_task_base.vertical_task->ref_count_t);
        fflush(stdout);
#endif
#ifdef TEMP_NESTED
#ifdef REUSE_GRAPH
        //dag reuse, restore ref_count_t
        if(((struct aceMesh_task*)self)->task_base.reused)
            ((struct aceMesh_task*)self)->task_base.ref_count_t=((struct aceMesh_task*)self)->task_base.backup_ref;
#endif
#endif
        return NULL;
    }
#ifdef ACEMESH_PARA
    mcur_time = rpcc();
    gid2 = (mcur_time - time_init - tt_build) / interval;
    //    printf("%d,%d, %ld, %ld, %ld\n",gid1, gid2,mcur_time,time_init,tt_build);
    if (gid1 < 0 || gid2 < 0 || gid2 < gid1) assert(0);
#endif
//    printf("user_execute_end\n");
    alog_debug("rank: %d,end ci_task_func", my_mpi_rank);
    return execute((struct aceMesh_task *)self);
}

#ifdef MPI_SURPPORT
inline void set_task_suspend(struct aceMesh_task *self, int mode)
{
    self->suspend = mode;
    alog_debug("set_task_suspend:%d\n", self->suspend);
}

void acemesh_set_suspend(int mode)
{
    set_task_suspend(&(cur_task->aceMesh_task_base), mode);
}
#ifdef MTEST_LIGHT
extern "C" void acemesh_set_mtest_handle(int *h1, int *h2, int kind1, int kind2)
{
    mtest_handle1 = h1;
    mtest_handle2 = h2;
    mtest_kind1 = kind1;
    mtest_kind2 = kind2;
}
#endif
#endif
#endif
// modify_control_thread_num
extern "C" void acemesh_set_thread_num(int thread_num) { sche_num_threads = thread_num; }

//extern "C" void acemesh_runtime_init(int total_threads)
void acemesh_runtime_init(int total_threads)
{
#ifdef THREAD_TRACE
    begin_time=rpcc();
    trace_print(0,'0',RTS_runtime_init);
    trace_num_threads=total_threads;
#endif
#ifdef MEMORY_POOL
slave_initpool();
    //printf("ci_task_size=%d\n",sizeof(ci_task));
#endif
#ifdef ACEMESH_LOG
    if (acelog_init("./log.conf") != 0) {
           exit(1);
    }
#endif
    cur_super_taskptr = NULL;
    if (total_threads == 0)
        aceMesh_runtime_init_zeroPara();
    else{
// #if defined(TARG_SW9) &&defined(CONCURRENT_CONSTRUCT_GRAPH)
//        if(total_threads<64)
//        aceMesh_runtime_init_TNum(total_threads);
//        else
//        aceMesh_runtime_init_TNum(total_threads-1);
// #else
        aceMesh_runtime_init_TNum(total_threads);
// #endif
    }    
    if (my_mpi_rank < 1) printf("thread_num:%d\n", total_threads);
#ifdef MEMORY_POOL
    InitPool();
    //slave_initpool();
#endif
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
    int i=0;
    for(i=0;i<N_BUILD_PROF;i++)
        buil_prof[i]=0;
#endif

#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_event_end);
#endif
    alog_info("acemesh init success");
}

extern "C" void acemesh_runtime_init_with_proc_id(int total_threads, int proc_id)
{
    aceMesh_MPI_rank(proc_id);
    acemesh_runtime_init(total_threads);
}

#ifdef ACEMESH_GRAPH_BUILD_PROFILING
char build_sec[N_BUILD_PROF][30]=
{
    "total_graph_build      ",
    "total_graph_exec       ",
    "total_time             ",
    "-begin_split           ",
    "-args_transmit         ",
    "-task_constructor      ",
    "-task_generator        ",
    "--register_call        ",
    "---add_end_task_list   ",
    "---build_relation      ",
    "----build_relat_in     ",
    "-----add_succ          ",
    "-----del_end_task_list ",
    "-----if-con(-)         ",
    "----find_hash          ",
    "----insert_hash        ",
    "----trave_list(-)      ",
    "----need_spawn_push    ",
    "--vec_clear            ",
    "-end_split             ",
    "--update_hashmap       ",
    "---update_item_in      ",
    "----find_hash          ",
    "----update_data        ",
#if defined(CONCURRENT_CONSTRUCT_GRAPH) && !defined(FAKE_CONCURRENT_GRAPH)
    "--clear_need_spawn     ",
#else
    "-clear_need_spawn      ",
#endif
    "-get_end_list          ",
    "-join_task             ",
    "-reset_graph           ",
    "-restore_graph         "
};
#endif

#ifdef ACEMESH_GRAPH_BUILD_PROFILING
int first_graph_build=1;
unsigned long graph_build_start=0;
unsigned long graph_build_end=0;
    #ifdef USE_COMPOSITE_TASK
unsigned long total_start=0;
    #endif
#endif

#ifdef EMP_MAIN_PUSH_PROFILING
extern int main_push_count;
extern unsigned long main_push_cycles;
extern unsigned long main_push_while_cycles;
#endif

#ifdef EMP_PUT_FULL_PROF
extern "C" unsigned gs_put_count;
extern "C" unsigned gs_put_full;
extern "C" unsigned ls_put_count[7];
extern "C" unsigned ls_put_full[7];
extern "C" unsigned worker_put_count[64];
extern "C" unsigned worker_put_full[64];
#endif

extern "C" void acemesh_runtime_shutdown()
{
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_runtime_shutdown);
#endif
    alog_info("acemesh shutdown");
    printf("acemesh shutdown\n");
#ifdef ACEMESH_LOG
    acelog_fini();
#endif
    aceMesh_runtime_shutdown();
    if(my_mpi_rank<=0){
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
    int i;
    std::cout << std::endl;
    std::cout << "---------Task graph Profiling---------" << std::endl;
    std::cout<<"graph_prof:"<<std::endl;
    for(i=0;i<N_BUILD_PROF;i++)
    {
        if(i<3)
        {
            printf("%s %7.4lf , %6.2lf%\n",build_sec[i],((double)buil_prof[i])/MFREQ,((double)buil_prof[i])/buil_prof[T_GRAPH_TIME]*100);
        }else if(i==IF_CON){
            //printf("%s %7.4lf , %6.2lf%\n",build_sec[i],((double)buil_prof[BUILD_RELAT]-buil_prof[BUILD_RELAT_IN])/MFREQ,((double)buil_prof[BUILD_RELAT]-buil_prof[BUILD_RELAT_IN])/buil_prof[0]*100);
            unsigned long tmp=buil_prof[BUILD_RELAT_IN]-buil_prof[ADD_SUCC]-buil_prof[DEL_END_L];
            printf("%s %7.4lf , %6.2lf%\n",build_sec[i],((double)tmp)/MFREQ,     ((double)tmp)/buil_prof[T_GRAPH_BUILD]*100);
        }else if(i==TRAVE_LIST){
            unsigned long tmp=buil_prof[BUILD_RELAT]-buil_prof[BUILD_RELAT_IN]-buil_prof[FIND_HASH]-buil_prof[INSERT_HASH]-buil_prof[NEED_SPAWN_P];
            printf("%s %7.4lf , %6.2lf%\n",build_sec[i],((double)tmp)/MFREQ,     ((double)tmp)/buil_prof[T_GRAPH_BUILD]*100);
        }else{
            printf("%s %7.4lf , %6.2lf%\n",build_sec[i],((double)buil_prof[i])/MFREQ,((double)buil_prof[i])/buil_prof[T_GRAPH_BUILD]*100);
        }
    }
#ifdef FINE_PROF2
    std::cout << "--------------------------------------" << std::endl;
    printf("find_hash_ntimes=%lu,each=%lu, find2=%lu,each=%lu\n",find_hash_ntimes1,buil_prof[FIND_HASH]/find_hash_ntimes1,find_hash_ntimes2,buil_prof[FIND_HASH2]/find_hash_ntimes2);
#endif
#if defined(ACEMESH_GRAPH_BUILD_PROFILING) && defined(FINE_PROF) && defined(REUSE_GRAPH)
    std::cout << "--------------------------------------" << std::endl;
    if(graph_reuse_times>0)
      std::cout << " reuse_times=" << graph_reuse_times<<" ,restore_graph overhead=" << (double)buil_prof[RESTROE_GRAPH]/graph_reuse_times/buil_prof[T_GRAPH_BUILD]*100 << " %" <<std::endl;
#endif
    std::cout << "--------------------------------------" << std::endl;
#endif

#ifdef TASK_LIFETIME_PROFILING
    unsigned long execute_cycles = 0, process_cycles = 0;
    int j;
    for (j = 0; j < sche_num_threads; j++) {
        execute_cycles += avr_execute_cycles[j];
        process_cycles += avr_process_cycles[j];
    }
    execute_cycles /= sche_num_threads;
    process_cycles /= sche_num_threads;
    float cycle_sum = avr_generate_cycles + execute_cycles + process_cycles;
    std::cout << std::endl;
    std::cout << "---------Task Lifetime Profiling---------" << std::endl;
    std::cout << "Task generate:\t" << avr_generate_cycles << "\tcycles" << std::endl;
    std::cout << "Task execute(cycles):\t" << execute_cycles << "\tcycles" << std::endl;
    std::cout << "Task process:\t" << process_cycles << "\tcycles" << std::endl;
    std::cout << std::endl;
    std::cout << "Task generate:\t" << avr_generate_cycles / cycle_sum * 100 << "%" << std::endl;
    std::cout << "Task execute:\t" << execute_cycles / cycle_sum * 100 << "%" << std::endl;
    std::cout << "Task process:\t" << process_cycles / cycle_sum * 100 << "%" << std::endl;
    std::cout << "-----------------------------------------" << std::endl;
    std::cout << std::endl;
#endif

#ifdef CONCURRENT_CONSTRUCT_GRAPH
#ifdef MAIN_MUTEX_PROFILING
    std::cout << "main core mutex fail count: " << fail_count << std::endl;
    std::cout << "main core mutex success count: " << success_count << std::endl;
#endif
#endif

#ifdef EMP_MAIN_PUSH_PROFILING
    std::cout << "EQ_main_push count: " << main_push_count << std::endl; 
    std::cout << "EQ_main_push time: " << main_push_cycles / MFREQ << std::endl;
    std::cout << "EQ_main_push while time: " << main_push_while_cycles / MFREQ << std::endl;
#endif
    }

#ifdef EMP_STEALING_PROF
    unsigned steal_fail_total = 0, steal_success_total = 0, steal_total = 0;
    for (int i = 0; i < 64; i++) {
        steal_fail_total += emp_stealing_fail[i];
        steal_success_total += emp_stealing_success[i];
    }
    steal_total = steal_fail_total + steal_success_total;
    std::cout << std::endl;
    std::cout << "---------EMP STEALING PROFILING----------" << std::endl;
    std::cout << "Total count: " << steal_total << std::endl;
    std::cout << "Success count: " << steal_success_total << std::endl;
    std::cout << "Fail count: " << steal_fail_total << std::endl;
    std::cout << "Success ratio: " << (double)steal_success_total / (double)steal_total * 100 << "%" << std::endl;
    std::cout << "-----------------------------------------" << std::endl;
    std::cout << std::endl;
#endif

#ifdef THREADS_IDLING_PROF
    unsigned total_get_count = 0, total_idle_count = 0, total_task_count;
    for (int i = 0; i < 64; i++) {
        total_get_count += thread_get_count[i];
        total_idle_count += thread_idle_count[i];
    }
    total_task_count = total_get_count - total_idle_count;
    std::cout << std::endl;
    std::cout << "--------THREADS IDLING PROFILING---------" << std::endl;
    std::cout << "Total get: " << total_get_count << std::endl;
    std::cout << "Total null: " << total_idle_count << std::endl;
    std::cout << "Total task got: " << total_task_count << std::endl;
    std::cout << "Idling ratio: " << (double)total_idle_count / (double)total_get_count * 100 << "%" << std::endl;
    std::cout << "-----------------------------------------" << std::endl;
    std::cout << std::endl;
#endif

#ifdef EMP_PUT_FULL_PROF
    unsigned total_ls_put_count = 0, total_ls_put_full = 0;
    unsigned total_worker_put_count = 0, total_worker_put_full = 0;
    for (int i = 0; i < 7; i++) {
        total_ls_put_count += ls_put_count[i];
        total_ls_put_full += ls_put_full[i];
    }

    for (int i = 0; i < 64; i++) {
        total_worker_put_count += worker_put_count[i];
        total_worker_put_full += worker_put_full[i];
    }

    std::cout << std::endl;
    std::cout << "-----------EMP-C Put Profiling-----------" << std::endl;
    std::cout << "Global Server:" << std::endl;
    std::cout << "put: " << gs_put_count << " buffer full: " << gs_put_full << std::endl;//" rate: " << (double)gs_put_full / (double)gs_put_count * 100 << "%" << std::endl;
    std::cout << "Local Server(total):" << std::endl;
    std::cout << "put: " << total_ls_put_count << " buffer full: " << total_ls_put_full << std::endl;//" rate: " << (double)total_ls_put_full / (double)total_ls_put_count * 100 << "%" << std::endl;
    std::cout << "Worker(total):" << std::endl;
    std::cout << "put: " << total_worker_put_count << " buffer full: " << total_worker_put_full << std::endl;//" rate: " << (double)total_worker_put_full / (double)total_worker_put_count * 100 << "%" << std::endl;
    std::cout << "-----------------------------------------" << std::endl;
    std::cout << std::endl;
#endif

#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_event_end);
    print_to_file(f,trace_num_threads);
    printf("total_number_of_master_trace:%d\n",total_m);
#endif

}
extern "C" void acemesh_runtime_shutdown_with_proc_id(int rankid)
{
    alog_info("acemesh shutdown");
#ifdef ACEMESH_LOG
    acelog_fini();
#endif
    acemesh_runtime_shutdown();

}

#ifdef SPECIFY_END_TASKS
extern "C" void acemesh_specify_end_tasks()
{
#ifdef TEMP_NESTED
    task_graphs[cur_nested_lev].acemesh_add_end_task(&(cur_task->aceMesh_task_base));
#else
    task_graph.acemesh_add_end_task(&(cur_task->aceMesh_task_base));
#endif
}
#endif

//extract common code from function
void _register_task(ci_task *_task, std::vector<addr_tuple> &local_vector, void *taskptr)
{
#ifdef THREAD_TRACE
   trace_time=rpcc()-begin_time;
   trace_print(trace_time,'0',RTS_register_task);
#endif


#ifdef TEMP_NESTED
    task_graphs[cur_nested_lev].register_task(&(_task->aceMesh_task_base), local_vector);
#else 
    if (taskptr) {
        task_dag_graph &nested_graph = ((aceMesh_composite_task *)taskptr)->graph;
        nested_graph.register_task(&(_task->aceMesh_task_base), local_vector);
    } else {
        task_graph.register_task(&(_task->aceMesh_task_base), local_vector);
    }
#endif


#ifdef BUFF_SEG
    int temp_affinity=(struct*)_task->affinity_id;
    if(temp_affinity<0 || temp_affinity>sche_num_threads || temp_affinity==sche_num_threads-1)
    {
      printf("error affinity\n");
      assert(0);
    }
#endif

#ifdef ACEMESH_SCHEDULER_PROFILING
    total_ntasks_register++;
#endif
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_event_end);
#endif
}
unsigned long task_gen_time=0,gen_temp=0;
void _task_gen(void *taskptr, ci_task *_task)
{
#ifdef NO_PARALLEL
    cur_task = _task;
#else
    (ci_taski *)&local_cur_task = &(cur_task.local());
    local_cur_task = task;
#endif

#ifdef SUPPORT_PARTITION
    _task->set_part_id(_sepid);
#endif
#ifdef NESTOPT
//gen_temp=rpcc();
   initTaskDataAccess(&(_task->aceMesh_task_base.task_base),local_addr.size());
    //struct TaskDataAccesses &local_access=getDataAccesses(&(_task->aceMesh_task_base.task_base));
//gen_temp=rpcc()-gen_temp;
//task_gen_time+=gen_temp;  
  for(std::tr1::unordered_map<void *, DataAccess>::iterator search_itr = local_addr.begin(); search_itr != local_addr.end(); ++search_itr){
      _task->aceMesh_task_base.task_base._dataAccesses._addressArray[_task->aceMesh_task_base.task_base._dataAccesses._currentIndex]=  search_itr->first;
    //    DataAccess *ret=&(local_access._accessArray[local_access._currentIndex++]);
    //     new (ret)DataAccess(search_itr->second._type,_task);
    _task->aceMesh_task_base.task_base._dataAccesses._accessArray[_task->aceMesh_task_base.task_base._dataAccesses._currentIndex]._type=search_itr->second._type;
    _task->aceMesh_task_base.task_base._dataAccesses._accessArray[_task->aceMesh_task_base.task_base._dataAccesses._currentIndex]._originator=&(_task->aceMesh_task_base.task_base);
    _task->aceMesh_task_base.task_base._dataAccesses._accessArray[_task->aceMesh_task_base.task_base._dataAccesses._currentIndex]._successor=nullptr;
_task->aceMesh_task_base.task_base._dataAccesses._accessArray[_task->aceMesh_task_base.task_base._dataAccesses._currentIndex].ishandle=0;   
 _task->aceMesh_task_base.task_base._dataAccesses._accessArray[_task->aceMesh_task_base.task_base._dataAccesses._currentIndex++]._child=nullptr;
       //local_access._currentIndex++;
    }
//gen_temp=rpcc()-gen_temp;
//task_gen_time+=gen_temp;
    // std::vector<addr_tuple> &local_vector = _v_addr_tuple.local();
#ifdef NO_PARALLEL
    std::vector<addr_tuple> &local_vector = _v_addr_tuple;
#else
    std::vector<addr_tuple> &local_vector = _v_addr_tuple.local();
#endif
#else
#ifdef NO_PARALLEL
    std::vector<addr_tuple> &local_vector = _v_addr_tuple;
#else
    std::vector<addr_tuple> &local_vector = _v_addr_tuple.local();
#endif
    //_task->aceMesh_task_base.loop_count = loop_id;  // add by gxr 2017/03/13

#endif
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
  #ifdef FINE_PROF
    unsigned long r_start=rpcc();
  #endif
#endif
    _register_task(_task, local_vector, taskptr);
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
  #ifdef FINE_PROF
    unsigned long r_end=rpcc();
    buil_prof[REG_TASK_IN]+=(r_end-r_start);
  #endif
#endif
    local_vector.clear();
    local_addr.clear();
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
  #ifdef FINE_PROF
    buil_prof[VEC_CLEAR]+=(rpcc()-r_end);
  #endif
#endif
}

void _task_gen_with_neighbors(void *taskptr, ci_task *_task)
{
#ifdef NO_PARALLEL
    cur_task = _task;
#else
    (ci_taski *)&local_cur_task = &(cur_task.local());
    local_cur_task = _task;
#endif
#ifdef SUPPORT_PARTITION
    _task->set_part_id(_sepid);
#endif

    addr_tuple item;
    item.addr = NULL;
    item.area_type = NORMAL;  //!!NOTE, same as UNSHADE, see comments in get_unshade_addr(),by chenli
    item.type = INOUT_NONE;
    item.neighbor = NEIGHBOR_ALL;
    item.neighbor_type = IN;
#ifdef NESTOPT
    int neighbor_num=0;
    void* neighbor_addrs[MAX_NEIGHBORS];
    get_neighbor(_task, item.neighbor, item.addr, neighbor_addrs, &neighbor_num);
    for(int i=0; i<neighbor_num; i++) {
        //void* neighbor_true_addr = task_graph.get_true_addr(neighbor_addrs[i], item.area_type);
    // printf("%d",neighbor_addrs.arr[i]);
    
      std::tr1::unordered_map<void *, DataAccess>::iterator search_itr = 
            local_addr.find(neighbor_addrs[i]);
            if(search_itr==local_addr.end()){
               struct DataAccess temp={nullptr,nullptr,nullptr,0,0,INOUT_NONE};
               local_addr.insert(std::pair<void*,DataAccess>(neighbor_addrs[i],temp));
            }
            else{
            //    if(search_itr!=local_addr.end()){
                int old_type=search_itr->second._type;
                if(old_type!=INOUT_NONE)
                search_itr->second._type=INOUT;
            //    }
            }
    }
    initTaskDataAccess(&(_task->aceMesh_task_base.task_base),local_addr.size());
    //struct TaskDataAccesses &local_access=getDataAccesses(&(_task->aceMesh_task_base.task_base));
    for(std::tr1::unordered_map<void *, DataAccess>::iterator search_itr = local_addr.begin(); search_itr != local_addr.end(); ++search_itr){
      _task->aceMesh_task_base.task_base._dataAccesses._addressArray[_task->aceMesh_task_base.task_base._dataAccesses._currentIndex]=  search_itr->first;
    //    DataAccess *ret=&(local_access._accessArray[local_access._currentIndex++]);
    //     new (ret)DataAccess(search_itr->second._type,_task);
    _task->aceMesh_task_base.task_base._dataAccesses._accessArray[_task->aceMesh_task_base.task_base._dataAccesses._currentIndex]._type=search_itr->second._type;
    _task->aceMesh_task_base.task_base._dataAccesses._accessArray[_task->aceMesh_task_base.task_base._dataAccesses._currentIndex]._originator=&(_task->aceMesh_task_base.task_base);
    _task->aceMesh_task_base.task_base._dataAccesses._accessArray[_task->aceMesh_task_base.task_base._dataAccesses._currentIndex]._successor=nullptr;
//acemesh_myalloc_aligned_64(sizeof(struct DataAccess*)*8);
_task->aceMesh_task_base.task_base._dataAccesses._accessArray[_task->aceMesh_task_base.task_base._dataAccesses._currentIndex].ishandle=0;
    _task->aceMesh_task_base.task_base._dataAccesses._accessArray[_task->aceMesh_task_base.task_base._dataAccesses._currentIndex++]._child=nullptr;
       //local_access._currentIndex++;
    }
std::vector<addr_tuple> &local_vector = _v_addr_tuple;
#else

#ifdef NO_PARALLEL
    std::vector<addr_tuple> &local_vector = _v_addr_tuple;
#else
    std::vector<addr_tuple> &local_vector = _v_addr_tuple.local();
#endif

    local_vector.push_back(item);
    //_task->aceMesh_task_base.loop_count = loop_id;  // add by gxr 2017/03/13
#endif
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
  #ifdef FINE_PROF
    unsigned long r_start=rpcc();
  #endif
#endif
    _register_task(_task, local_vector, taskptr);
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
  #ifdef FINE_PROF
    unsigned long r_end=rpcc();
    buil_prof[REG_TASK_IN]+=(r_end-r_start);
  #endif
#endif
#ifdef NESTOPT
local_addr.clear();
#else
    local_vector.clear();
#endif
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
  #ifdef FINE_PROF
    buil_prof[VEC_CLEAR]+=(rpcc()-r_end);
  #endif
#endif
}

extern "C" void acemesh_task_generator(void *taskfptr, void *args, unsigned int args_size)
{
#ifdef TASK_LIFETIME_PROFILING
    unsigned long gene_start = rpcc();
#endif

#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    unsigned long b_start=rpcc();
#endif
#endif
    TASK_FUNCPTR funcptr = (TASK_FUNCPTR)taskfptr;
    ci_task *_task;
#ifdef MEMORY_POOL
    alog_debug("sizeof comptask: %u", sizeof(ci_task));
    _task = (ci_task *)acemesh_myalloc_aligned_16(sizeof(ci_task));
    alog_debug("addr of _task: %x", _task);
#else
#ifdef TARG_SW5
    _task = (ci_task *)malloc(sizeof(ci_task));
#elif TARG_SW9
    //_task = (ci_task *)libc_uncached_aligned_malloc(sizeof(ci_task));
    _task = (ci_task *)my_malloc_aligned(sizeof(ci_task));
#else
    assert(0);
#endif
#ifdef DEBUG
    printf("generator_task=%p\n",_task);
    fflush(stdout);
#endif
#endif
    ci_task_constructor(_task, funcptr);
    define(_task, args, args_size);
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    unsigned long b_end=rpcc();
    buil_prof[TASK_CONSTR]+=(b_end-b_start);
#endif
#endif
    _task_gen(cur_super_taskptr, _task);
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    buil_prof[TASK_GENE]+=(rpcc()-b_end);
#endif
#endif


#ifdef TASK_LIFETIME_PROFILING
    unsigned long gene_end = rpcc();
    if (avr_generate_cycles != 0) {
        avr_generate_cycles += (gene_end - gene_start);
        avr_generate_cycles /= 2;
    }
    else {
        avr_generate_cycles = gene_end - gene_start;
    }
#endif

}

extern "C" void acemesh_composite_task_generator(void *taskfptr, void *args, unsigned int args_size)
{
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    unsigned long b_start=rpcc();
#endif
#endif
    TASK_FUNCPTR funcptr = (TASK_FUNCPTR)taskfptr;
    alog_debug("funcptr: %x", taskfptr);
    ci_task *_task;
#ifdef MEMORY_POOL
    alog_debug("sizeof comptask: %u", sizeof(aceMesh_composite_task));
    _task = (ci_task *)acemesh_myalloc_aligned_16(sizeof(aceMesh_composite_task));
    alog_debug("addr of _task: %x", _task);
#else
#ifdef TARG_SW5
    _task = (ci_task *)malloc(sizeof(aceMesh_composite_task));
#elif TARG_SW9
    //_task = (ci_task *)libc_uncached_aligned_malloc(sizeof(aceMesh_composite_task));
    _task = (ci_task *)my_malloc_aligned(sizeof(aceMesh_composite_task));
#else
    assert(0);
#endif
#endif
    //printf("composite_generator_task=%p,%d\n",_task,args_size);
    //fflush(stdout);
    aceMesh_composite_task_constructor((aceMesh_composite_task* )_task, funcptr);
    
    define_with_taskptr(_task, args, args_size, _task);

#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    unsigned long b_end=rpcc();
    buil_prof[TASK_CONSTR]+=(b_end-b_start);
#endif
#endif
    _task_gen(cur_super_taskptr, _task);
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    b_start=rpcc();
    buil_prof[TASK_GENE]+=(b_start-b_end);
#endif
#endif

}

extern "C" void acemesh_task_generator_with_neighbors(void *taskfptr, void *args,
                                                      unsigned int args_size,
                                                      void *cxx_this_pointer,
                                                      void *get_neighbors_funcptr,
                                                      void *neighbor_args)
{
#ifdef TASK_LIFETIME_PROFILING
    unsigned long gene_start = rpcc();
#endif

#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    unsigned long b_start=rpcc();
#endif
#endif

    TASK_FUNCPTR funcptr = (TASK_FUNCPTR)taskfptr;

    ci_task *_task;
#ifdef MEMORY_POOL
    alog_debug("sizeof comptask: %u", sizeof(ci_task));
    _task = (ci_task *)acemesh_myalloc_aligned_16(sizeof(ci_task));
    alog_debug("addr of _task: %x", _task);
#else
#ifdef TARG_SW5
    _task = (ci_task *)malloc(sizeof(ci_task));
#elif TARG_SW9
    //_task = (ci_task *)libc_uncached_aligned_malloc(sizeof(ci_task));
    _task = (ci_task *)my_malloc_aligned(sizeof(ci_task));
#else
    assert(0);
#endif
#endif
#ifdef DEBUG
    printf("create_task=%p\n",_task);
    printf("generator_with_neighbor_task=%p\n",_task);
    fflush(stdout);
#endif

    ci_task_constructor(_task, funcptr);
    define(_task, args, args_size);
    define_neighbors(_task, cxx_this_pointer, get_neighbors_funcptr, neighbor_args);
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    unsigned long b_end=rpcc();
    buil_prof[TASK_CONSTR]+=(b_end-b_start);
#endif
#endif

    _task_gen_with_neighbors(cur_super_taskptr, _task);
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    b_start=rpcc();
    buil_prof[TASK_GENE]+=(b_start-b_end);
#endif
#endif


#ifdef TASK_LIFETIME_PROFILING
    unsigned long gene_end = rpcc();
    if (avr_generate_cycles != 0) {
        avr_generate_cycles += (gene_end - gene_start);
        avr_generate_cycles /= 2;
    }
    else {
        avr_generate_cycles = gene_end - gene_start;
    }
#endif

}



#ifdef DOUBLE_FUNC

extern "C" void acemesh_task_generator_msfunc(void *mtaskfptr, void *staskfptr, void *args, unsigned int args_size)
{
#ifdef TASK_LIFETIME_PROFILING
    unsigned long gene_start = rpcc();
#endif

#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    unsigned long b_start=rpcc();
#endif
#endif
    TASK_FUNCPTR mfuncptr = (TASK_FUNCPTR)mtaskfptr;
    TASK_FUNCPTR sfuncptr = (TASK_FUNCPTR)staskfptr;
    ci_task *_task;
#ifdef MEMORY_POOL
    alog_debug("sizeof comptask: %u", sizeof(ci_task));
    _task = (ci_task *)acemesh_myalloc_aligned_16(sizeof(ci_task));
    alog_debug("addr of _task: %x", _task);
#else
#ifdef TARG_SW5
    _task = (ci_task *)malloc(sizeof(ci_task));
#elif TARG_SW9
    //_task = (ci_task *)libc_uncached_aligned_malloc(sizeof(ci_task));
    _task = (ci_task *)my_malloc_aligned(sizeof(ci_task));
#else
    assert(0);
#endif
#endif
    ci_task_constructor_msfunc(_task, mfuncptr, sfuncptr);
    define(_task, args, args_size);
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    unsigned long b_end=rpcc();
    buil_prof[TASK_CONSTR]+=(b_end-b_start);
#endif
#endif
    _task_gen(cur_super_taskptr, _task);
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    b_start=rpcc();
    buil_prof[TASK_GENE]+=(b_start-b_end);
#endif
#endif


#ifdef TASK_LIFETIME_PROFILING
    unsigned long gene_end = rpcc();
    if (avr_generate_cycles != 0) {
        avr_generate_cycles += (gene_end - gene_start);
        avr_generate_cycles /= 2;
    }
    else {
        avr_generate_cycles = gene_end - gene_start;
    }
#endif

}



extern "C" void acemesh_task_generator_with_neighbors_msfunc(void *mtaskfptr, void *staskfptr, void *args,
                                                      unsigned int args_size,
                                                      void *cxx_this_pointer,
                                                      void *get_neighbors_funcptr,
                                                      void *neighbor_args)
{
#ifdef TASK_LIFETIME_PROFILING
    unsigned long gene_start = rpcc();
#endif

#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    unsigned long b_start=rpcc();
#endif
#endif

    TASK_FUNCPTR mfuncptr = (TASK_FUNCPTR)mtaskfptr;
    TASK_FUNCPTR sfuncptr = (TASK_FUNCPTR)staskfptr;

    ci_task *_task;
#ifdef MEMORY_POOL
    alog_debug("sizeof comptask: %u", sizeof(ci_task));
    _task = (ci_task *)acemesh_myalloc_aligned_16(sizeof(ci_task));
    alog_debug("addr of _task: %x", _task);
#else
#ifdef TARG_SW5
    _task = (ci_task *)malloc(sizeof(ci_task));
#elif TARG_SW9
    //_task = (ci_task *)libc_uncached_aligned_malloc(sizeof(ci_task));
    _task = (ci_task *)my_malloc_aligned(sizeof(ci_task));
#else
    assert(0);
#endif
#endif

    ci_task_constructor_msfunc(_task, mfuncptr, sfuncptr);
    define(_task, args, args_size);
    define_neighbors(_task, cxx_this_pointer, get_neighbors_funcptr, neighbor_args);
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    unsigned long b_end=rpcc();
    buil_prof[TASK_CONSTR]+=(b_end-b_start);
#endif
#endif

    _task_gen_with_neighbors(cur_super_taskptr, _task);
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    b_start=rpcc();
    buil_prof[TASK_GENE]+=(b_start-b_end);
#endif
#endif


#ifdef TASK_LIFETIME_PROFILING
    unsigned long gene_end = rpcc();
    if (avr_generate_cycles != 0) {
        avr_generate_cycles += (gene_end - gene_start);
        avr_generate_cycles /= 2;
    }
    else {
        avr_generate_cycles = gene_end - gene_start;
    }
#endif

}

#endif



static void acemesh_do_push(int argc, void *addr, int area_flag, va_list *p_args, int type)
{
#ifdef THREAD_TRACE_LEVEL_1
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_push_addr);
#endif
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    unsigned long push_start = rpcc();
#endif
#endif
#ifdef NESTOPT
// std::pair<std::tr1::unordered_map<void *, DataAccess>::iterator, bool> emplaced = local_addr.emplace(std::piecewise_construct,
// 				std::forward_as_tuple(addr),
// 				std::forward_as_tuple(type, nullptr));//插入失败，表示存在address
printf("do_push:%p\n",addr);            
std::tr1::unordered_map<void *, DataAccess>::iterator search_itr = 
            local_addr.find(addr);
            if(search_itr==local_addr.end()){
               struct DataAccess temp={nullptr,nullptr,nullptr,0,0,type};
               local_addr.insert(std::pair<void*,DataAccess>(addr,temp));
            }
            else{
            //    if(search_itr!=local_addr.end()){
                int old_type=search_itr->second._type;
                if(old_type!=type)
                search_itr->second._type=INOUT;
            //    }
            }
while(argc-1){ 
    // std::pair<std::tr1::unordered_map<void *, DataAccess>::iterator, bool> emplaced = local_addr.emplace(std::piecewise_construct,
	// 			std::forward_as_tuple(va_arg(*p_args, void *)),
	// 			std::forward_as_tuple(va_arg(*p_args, int), nullptr));//插入失败，表示存在address
    //         if(!emplaced.second){  //update type
            std::tr1::unordered_map<void *, DataAccess>::iterator search_itr = 
            local_addr.find(va_arg(*p_args, void *));
            if(search_itr==local_addr.end()){
               local_addr[va_arg(*p_args, void *)]._type=va_arg(*p_args, int);
            }
            else{
                int old_type=search_itr->second._type;
                if(old_type!=va_arg(*p_args, int))
                search_itr->second._type=INOUT;
               }
            
    --argc;
}
#else
    addr_tuple item;

    item.type = type;
    item.neighbor = NEIGHBOR_NONE;
    item.neighbor_type = INOUT_NONE;

    item.addr = addr;
    item.area_type = area_flag;

#ifdef NO_PARALLEL
    std::vector<addr_tuple> &local_vector = _v_addr_tuple;
#else
    std::vector<addr_tuple> &local_vector = _v_addr_tuple.local();
#endif

//#ifdef LIGHT_BUILD_GRAPH
//    local_vector.push_back(item);
//#else
    if (area_flag == SHADE_AND_UNSHADE) {
        item.area_type = SHADE;
        local_vector.push_back(item);

        item.area_type = UNSHADE;
        local_vector.push_back(item);
    } else {
        local_vector.push_back(item);
    }
//#endif

    while (argc - 1) {
        item.addr = va_arg(*p_args, void *);
        item.area_type = va_arg(*p_args, int);
//#ifdef LIGHT_BUILD_GRAPH
//    local_vector.push_back(item);
//#else
        if (item.area_type == SHADE_AND_UNSHADE) {
            item.area_type = SHADE;
            local_vector.push_back(item);

            item.area_type = UNSHADE;
            local_vector.push_back(item);
        } else {
            local_vector.push_back(item);
        }
//#endif
        --argc;
    }
#endif
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    buil_prof[ARGS_TRANS]+= (rpcc()-push_start);
#endif
#endif
#ifdef THREAD_TRACE_LEVEL_1
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_event_end);
#endif
}

static void acemesh_do_push1(int argc, void *addr, int area_flag, int type)
{
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
        unsigned long push_start = rpcc();
#endif
#endif
    addr_tuple item;

    item.type = type;
    item.neighbor = NEIGHBOR_NONE;
    item.neighbor_type = INOUT_NONE;

    item.addr = addr;
    item.area_type = area_flag;

#ifdef NO_PARALLEL
    std::vector<addr_tuple> &local_vector = _v_addr_tuple;
#else
    std::vector<addr_tuple> &local_vector = _v_addr_tuple.local();
#endif

//#ifdef LIGHT_BUILD_GRAPH
//    local_vector.push_back(item);
//#else
    if (area_flag == SHADE_AND_UNSHADE) {
        item.area_type = SHADE;
        local_vector.push_back(item);

        item.area_type = UNSHADE;
        local_vector.push_back(item);
    } else {
        local_vector.push_back(item);
    }
//#endif
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
        buil_prof[ARGS_TRANS]+= (rpcc()-push_start);
#endif
#endif
}

extern "C" void acemesh_push_wrlist(int argc, void *addr, int area_flag, ...)
{
    va_list args;
    va_start(args, area_flag);
    acemesh_do_push(argc, addr, area_flag, &args, INOUT);
    va_end(args);
}

extern "C" void acemesh_push_rlist(int argc, void *addr, int area_flag, ...)
{
    va_list args;
    va_start(args, area_flag);
    acemesh_do_push(argc, addr, area_flag, &args, IN);
    va_end(args);
}

extern "C" void acemesh_push_wlist(int argc, void *addr, int area_flag, ...)
{
    va_list args;
    va_start(args, area_flag);
    acemesh_do_push(argc, addr, area_flag, &args, OUT);
    va_end(args);
}

//extern "C" void acemesh_begin_split_task(char *loop_info)
void acemesh_begin_split_task(char *loop_info)
{
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_begin_split_task);
#endif
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
    unsigned long split_task_start=rpcc();
#endif
    if(first_graph_build==1){
        graph_build_start=rpcc();
    #ifdef USE_COMPOSITE_TASK
        if (!total_start)
            total_start=rpcc();
    #endif
        first_graph_build=0;
    }
#endif
    alog_debug("cur_super_taskptr: %x",cur_super_taskptr);
    if ((loop_info == 0) || (*loop_info == '\0')) {
        begin_split_task_zeroPara(cur_super_taskptr);
    } else {
        ++loop_id;
        begin_split_task_charPara(loop_info, cur_super_taskptr);
    }
#ifdef ACEMESH_GRAPH_BUILD_PROFILING
#ifdef COARSE_PROF
     buil_prof[BEGIN_SPLIT]+=( rpcc()-split_task_start);
#endif
#endif
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_event_end);
#endif
}
extern "C" void acemesh_end_split_task() { 
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_end_split_task);
#endif
#if defined( ACEMESH_GRAPH_BUILD_PROFILING) && defined(COARSE_PROF)
    unsigned long split_task_start=rpcc();
#endif
    alog_debug("cur_super_taskptr: %x",cur_super_taskptr);
    end_split_task_zeroPara(cur_super_taskptr);
#if defined( ACEMESH_GRAPH_BUILD_PROFILING) && defined(COARSE_PROF)
    buil_prof[END_SPLIT]+=( rpcc()-split_task_start);
#endif
#ifdef THREAD_TRACE
    trace_time=rpcc()-begin_time;
    trace_print(trace_time,'0',RTS_event_end);
#endif
}
static std::vector<void *> enlonged_local_objs;
extern "C" void *acemesh_malloc_obj(unsigned int args_size)
{
    void *addr = malloc(args_size);
    enlonged_local_objs.push_back(addr);
    return addr;
}
static void acemesh_free_all_objs(void)
{
    std::vector<void *>::iterator itr;
    for (itr = enlonged_local_objs.begin(); itr != enlonged_local_objs.end(); ++itr) {
        void *addr = *itr;
        free(addr);
    }
    enlonged_local_objs.clear();
}

//#ifdef ACEMESH_PROFILING
extern "C" unsigned long acmemesh_master_rpcc(unsigned long* t_count)
{
    *t_count=rpcc();
    return rpcc();
}
extern "C" unsigned long acmemesh_master_rpcc_(unsigned long* t_count)
{
    *t_count=rpcc();
    return rpcc();
}
//#endif
extern "C" void acemesh_spawn_and_wait(int print_graph)
{
    alog_debug("cur_super_taskptr: %x", cur_super_taskptr);
    //printf("cur_super_taskptr: %x\n", cur_super_taskptr);
    //fflush(stdout);
    if (!cur_super_taskptr) {
#ifdef ACEMESH_PROFILING_CLASSIFY
        spawn_and_wait(print_graph,cur_super_taskptr);
#else
        if (print_graph == 0)
            spawn_and_wait_zeroPara(cur_super_taskptr);
        else
            spawn_and_wait(print_graph, cur_super_taskptr);
#endif
        acemesh_free_all_objs();
#ifdef MEMORY_POOL
        ReInitial();
#endif
    } else {
        spawn_and_goout(cur_super_taskptr);
    }
}

extern "C" void acemesh_spawn_and_goout()
{
    spawn_and_goout(cur_super_taskptr);
}

extern "C" void acemesh_spawn_and_goout_()
{
    spawn_and_goout(cur_super_taskptr);
}

#ifdef REUSE_GRAPH
extern "C" int acemesh_dag_start_vec(int dagNo, int *int_vec, int n1, double *float_vec, int n2)
  {
    return dag_start(dagNo, int_vec, n1, float_vec, n2);
}
extern "C" int acemesh_dag_start(int dagNo) { return acemesh_dag_start_vec(dagNo, 0, 0, 0, 0); }
#endif

extern "C" int acemesh_get_loop_tile_num(int loop_tile_size, int loop_index_lb, int loop_index_ub)
{
    int loop_tile_num;
    int loop_index_len;
    int lb_mod;
    int ub_mod;

    loop_tile_num = 0;
    if(g_init_offset==0)
    {    

    if ((loop_index_ub - loop_index_lb >= 0 && loop_tile_size > 0) ||
        (loop_index_ub - loop_index_lb <= 0 && loop_tile_size < 0)) {
        if (loop_index_ub - loop_index_lb >= 0)
            loop_index_len = loop_index_ub - loop_index_lb + 1;
        else {
            loop_index_len = loop_index_ub - loop_index_lb - 1;

            loop_index_len = loop_index_len * (-1);
            loop_tile_size = loop_tile_size * (-1);
        }
        // pre loop
        lb_mod = loop_index_lb % loop_tile_size;
        if (lb_mod != 0) {
            loop_tile_num = loop_tile_num + 1;
            if (loop_index_len < (loop_tile_size - lb_mod))
                loop_index_len = 0;
            else
                loop_index_len = loop_index_len - (loop_tile_size - lb_mod);
        }

        // post loop
        ub_mod = loop_index_ub % loop_tile_size;
        if ((ub_mod != (loop_tile_size - 1)) && (loop_index_len > loop_tile_size)) {
            loop_tile_num = loop_tile_num + 1;
            loop_index_len = loop_index_len - (ub_mod + 1);
        }
        // main loop
        if (loop_index_len > 0) {
            if (loop_index_len < loop_tile_size)
                loop_tile_num = loop_tile_num + 1;
            else
                loop_tile_num = loop_tile_num + loop_index_len / loop_tile_size;
        }
    }
    }
    else
    {
#ifdef OLDOFFSET
    int lno_1, lno_2;
    if (loop_index_lb > loop_index_ub) return 0;
    lno_1 = (loop_index_ub - g_init_offset) / loop_tile_size;
    lno_2 = (loop_index_lb - g_init_offset) / loop_tile_size;

    loop_tile_num = lno_1 - lno_2 + 1;
#else
      int new_lb_index=0;
      int new_ub_index=0;
      if (loop_index_lb > loop_index_ub) return -1;
      //int local_lb_index=(loop_index_lb%loop_tile_size+g_init_offset%loop_tile_size)%loop_tile_size;
      int local_lb_index=((loop_index_lb-g_init_offset)%loop_tile_size+loop_tile_size)%loop_tile_size;
      if(local_lb_index!=0)
      {   
        loop_tile_num+=1;
        new_lb_index=loop_index_lb+(loop_tile_size-local_lb_index);
      }   
      else
        new_lb_index=loop_index_lb;

      //int local_ub_index=(loop_index_ub%loop_tile_size+g_init_offset%loop_tile_size)%loop_tile_size;
      int local_ub_index=((loop_index_ub-g_init_offset)%loop_tile_size+loop_tile_size)%loop_tile_size;
      if(local_ub_index!=loop_tile_size-1)
      {   
        loop_tile_num+=1;
        new_ub_index=loop_index_ub-local_ub_index-1;
      }   
      else
        new_ub_index=loop_index_ub;

      loop_tile_num+=ceil((double)(new_ub_index-new_lb_index+1)/loop_tile_size);
#endif
    }
    return loop_tile_num;
}

extern "C" int acemesh_get_tile_no_by_index(int tile_size, int compute_index, int index_lb)
{
//    int tile_no;
//    tile_no = (compute_index - (index_lb - index_lb % tile_size)) / tile_size;
    int lno_1, lno_2, ret;
    lno_1 = (compute_index - g_init_offset % tile_size) / tile_size;
    lno_2 = (index_lb - g_init_offset % tile_size) / tile_size;
    ret = lno_1 - lno_2 ;

    return ret;
}

extern "C" int acemesh_get_loop_tile_start_index_by_tile_no(int loop_tile_size, int loop_tile_no,
                                                            int loop_index_lb)
{
//    int ret = loop_tile_no * loop_tile_size + loop_index_lb - (loop_index_lb % loop_tile_size);
//    if (ret < loop_index_lb)
//        return loop_index_lb;
//    else
//        return ret;
    int ret = (loop_tile_no + (loop_index_lb - g_init_offset) / loop_tile_size) * loop_tile_size + g_init_offset;
    if (ret < loop_index_lb)
        return loop_index_lb;
    else
        return ret;
}
extern "C" int acemesh_get_loop_tile_start_index_by_tile_no_with_ub(int loop_tile_size, int loop_tile_no,
                                                            int loop_index_lb, int loop_index_ub)
{
#ifdef OLDOFFSET
    int ret = (loop_tile_no + (loop_index_lb - g_init_offset) / loop_tile_size) * loop_tile_size + g_init_offset;
#else
    //int local_lb_index=(loop_index_lb%loop_tile_size+g_init_offset%loop_tile_size)%loop_tile_size;
    int local_lb_index=((loop_index_lb-g_init_offset)%loop_tile_size+loop_tile_size)%loop_tile_size;
    int new_lb_index=loop_index_lb-local_lb_index;
    int ret=new_lb_index+loop_tile_no*loop_tile_size;
#endif
    if (ret > loop_index_ub)
        return loop_index_ub; 
    else if (ret < loop_index_lb)
        return loop_index_lb;
    else
        return ret;
}

extern "C" int acemesh_get_loop_tile_end_index_by_tile_no(int loop_tile_size, int loop_tile_no,
                                                          int loop_index_lb, int loop_index_ub)
{
//    int ret;
//    ret = (loop_tile_no + 1) * loop_tile_size + loop_index_lb - (loop_index_lb % loop_tile_size) - 1;
//    if (ret > loop_index_ub)
//        return loop_index_ub;
//    else
//        return ret;
#ifdef OLDOFFSET
    int ret;
    ret = (loop_tile_no + (loop_index_lb - g_init_offset) / loop_tile_size + 1) * loop_tile_size + g_init_offset - 1;
#else
    int local_lb_index=((loop_index_lb-g_init_offset)%loop_tile_size+loop_tile_size)%loop_tile_size;
    int new_lb_index=loop_index_lb-local_lb_index;
    int ret=new_lb_index+(loop_tile_no+1)*loop_tile_size-1;
#endif
    if (ret > loop_index_ub)
        return loop_index_ub;
    else if((ret < loop_index_lb))
        return loop_index_lb;
    else
        return ret;
}

extern "C" int acemesh_get_data_tile_start_index_by_index(int data_tile_size, int compute_index,
                                                          int array_dim_lb)
{
    int ret;
//    ret = compute_index - (compute_index % data_tile_size);
//    if (ret < array_dim_lb)
//        return array_dim_lb;
//    else
//        return ret;
#ifdef OLDOFFSET
  ret = (compute_index-g_init_offset)/data_tile_size*data_tile_size+g_init_offset;
#else
//    int local_lb_index=(compute_index%data_tile_size+g_init_offset%data_tile_size)%data_tile_size;

    int local_lb_index=((compute_index-g_init_offset)%data_tile_size+data_tile_size)%data_tile_size;
    ret=compute_index-local_lb_index;
#endif
  if (ret < array_dim_lb)
    return array_dim_lb;
  else
    return ret;
}
extern "C" int acemesh_get_loop_tile_num_with_init_offset(int loop_tile_size, int loop_index_lb,
                                                          int loop_index_ub, int init_offset)
{
    int loop_tile_num = 0;
    int lno_1, lno_2;
    if (loop_index_lb > loop_index_ub) return 0;
    // if((loop_index_lb<loop_index_ub && loop_tile_size > 0)||(loop_index_lb>loop_index_ub &&
    // loop_tile_size < 0))
    //{
    lno_1 = (loop_index_ub - init_offset) / loop_tile_size;
    lno_2 = (loop_index_lb - init_offset) / loop_tile_size;

    loop_tile_num = lno_1 - lno_2 + 1;
    //}
    // std::cout << loop_tile_num << loop_tile_size << loop_index_lb << loop_index_ub << init_offset
    // << lno_1 << lno_2 << std::endl;
    return loop_tile_num;
}
extern "C" int acemesh_get_loop_tile_start_index_by_tile_no_with_init_offset(int loop_tile_size,
                                                                             int loop_tile_no,
                                                                             int loop_index_lb,
                                                                             int init_offset)
{
    int ret = (loop_tile_no + (loop_index_lb - init_offset) / loop_tile_size) * loop_tile_size +
              init_offset;
    // std::cout <<  loop_tile_size << loop_tile_no << loop_index_lb << init_offset << ret <<
    // std::endl;
    if (ret < loop_index_lb)
        return loop_index_lb;
    else
        return ret;
}
extern "C" int acemesh_get_loop_tile_end_index_by_tile_no_with_init_offset(
    int loop_tile_size, int loop_tile_no, int loop_index_lb, int loop_index_ub, int init_offset)
{
    int ret;
    ret = (loop_tile_no + (loop_index_lb - init_offset) / loop_tile_size + 1) * loop_tile_size +
          init_offset - 1;
    // ret = (loop_tile_no+1)* loop_tile_size - init_offset % loop_tile_size + loop_index_lb -
    // (loop_index_lb % loop_tile_size)-1;
    if (ret > loop_index_ub)
        return loop_index_ub;
    else
        return ret;
}
extern "C" int acemesh_get_data_tile_start_index_by_index_with_init_offset(int data_tile_size,
                                                                           int compute_index,
                                                                           int array_dim_lb,
                                                                           int init_offset)
{
    int ret;
    ret = (compute_index - init_offset) / data_tile_size * data_tile_size + init_offset;
    //  ret = compute_index - ( (compute_index -init_offset % data_tile_size)% data_tile_size);
    if (ret < array_dim_lb)
        return array_dim_lb;
    else
        return ret;
}

extern "C" int acemesh_get_tile_no_by_index_with_init_offset(int tile_size, int index, int lb,
                                                             int init_offset)
{
    int lno_1, lno_2, ret;
    lno_1 = (index - init_offset % tile_size) / tile_size;
    lno_2 = (lb - init_offset % tile_size) / tile_size;
    ret = lno_1 - lno_2;
    return ret;
}

extern "C" void acemesh_task_generator_swf2(void *taskfptr, void *args, unsigned int args_size)
{
    acemesh_task_generator(taskfptr, args, args_size);
}

extern "C" void acemesh_task_generator_with_neighbors_swf2(void *task_funcptr, void *args,
                                                           unsigned int args_size,
                                                           void *cxx_this_pointer,
                                                           void *neighbor_task_funcptr,
                                                           void *neighbor_args)
{
    acemesh_task_generator_with_neighbors(task_funcptr, args, args_size, cxx_this_pointer,
                                          neighbor_task_funcptr, neighbor_args);
}

#ifdef SUPPORT_PARTITION
void acemesh_setsepid(int id) { _sepid = id; }
#endif

extern "C" void acemesh_push_neighbor_addr(void *addr) { 
#ifdef TEMP_NESTED
    task_graphs[cur_nested_lev].push_neighbor_addr(addr);
#else
    if (cur_super_taskptr) {
        task_dag_graph &nested_graph = ((aceMesh_composite_task *)cur_super_taskptr)->graph;
        nested_graph.push_neighbor_addr(addr);
    } else {
        task_graph.push_neighbor_addr(addr);
    }
#endif
}

#ifdef SWF

#ifdef ACEMESH_PARA
extern int interval;

extern "C" void acemesh_set_interval_(int it) { acemesh_set_interval(it); }
#endif

extern "C" void acemesh_task_set_type_(task_type type) { acemesh_task_set_type(type); }

extern "C" void acemesh_task_set_affinity_(int id) { acemesh_task_set_affinity(id); }

extern "C" void acemesh_set_thread_num_(int thread_num) { acemesh_set_thread_num(thread_num); }

#if defined(USE_PRIORITY_QUEUE) || defined(LOCAL_MULTI_PRIORI)
extern "C" void acemesh_task_set_priority_id_(int id) { acemesh_task_set_priority_id(id); }
extern "C" int acemesh_task_get_priority_id_() { return acemesh_task_get_priority_id(); }
#endif

extern "C" void acemesh_runtime_init_(int total_threads) { acemesh_runtime_init(total_threads); }

extern "C" void acemesh_runtime_init_with_proc_id_(int total_threads, int proc_id)
{
    acemesh_runtime_init_with_proc_id(total_threads, proc_id);
}

extern "C" void acemesh_runtime_shutdown_() { acemesh_runtime_shutdown(); }
extern "C" void acemesh_runtime_shutdown_with_proc_id_(int proc_id)
{
    printf("acemesh_clang.cpp shutdown\n");
    acemesh_runtime_shutdown_with_proc_id(proc_id);
}
extern "C" void acemesh_push_rlist1_(int argc, void *addr, int area_flag)
{
    acemesh_do_push1(argc, addr, area_flag, IN);
}

extern "C" void acemesh_push_wrlist1_(int argc, void *addr, int area_flag)
{
    acemesh_do_push1(argc, addr, area_flag, INOUT);
}
extern "C" void acemesh_push_wlist1_(int argc, void *addr, int area_flag)
{
    acemesh_do_push1(argc, addr, area_flag, OUT);
}
extern "C" void acemesh_set_mod_off(int mod_off)
{
   g_init_offset = mod_off;
   //std::cout<<"init offset is :"<<g_init_offset<<std::endl;
}
extern "C" void acemesh_set_mod_off_(int mod_off)
{
   acemesh_set_mod_off(mod_off);
}
extern "C" void acemesh_runtime_begin()
{

}
extern "C" void acemesh_runtime_begin_()
{
   acemesh_runtime_begin();
}

#ifdef AUTO_AFFINITY
extern "C" void acemesh_reset_affinity_() { acemesh_reset_affinity(); }
#endif
extern "C" void acemesh_begin_split_task_(char *loop_info) { acemesh_begin_split_task(loop_info); }
extern "C" void acemesh_end_split_task_() { acemesh_end_split_task(); }
extern "C" void acemesh_task_generator_swf_(int task_funcno, void *args, unsigned int args_size)
{
#ifdef OLD_STYLE
    acemesh_task_generator(get_taskfunc_pointer(task_funcno), args, args_size);
#else
    assert(0);
#endif
}

extern "C" void acemesh_task_generator_swf2_(void *taskfptr, void *args, unsigned int args_size)
{
    acemesh_task_generator(taskfptr, args, args_size);
}

extern "C" void acemesh_composite_task_generator_(void *taskfptr, void *args, unsigned int args_size)
{
    acemesh_composite_task_generator(taskfptr, args, args_size);
}

#ifdef SPECIFY_END_TASKS
extern "C" void acemesh_specify_end_tasks_() { acemesh_specify_end_tasks(); }
#endif

extern "C" void acemesh_task_generator_with_neighbors_swf_(int task_funcno, void *args,
                                                           unsigned int args_size,
                                                           void *cxx_this_pointer,
                                                           int neighbor_task_funcno,
                                                           void *neighbor_args)
{
#ifdef OLD_STYLE
    acemesh_task_generator_with_neighbors(
        get_taskfunc_pointer(task_funcno), args, args_size, cxx_this_pointer,
        get_taskfunc_pointer(neighbor_task_funcno), neighbor_args);
#else
    assert(0);
#endif
}

extern "C" void acemesh_task_generator_with_neighbors_swf2_(void *task_funcptr, void *args,
                                                            unsigned int args_size,
                                                            void *cxx_this_pointer,
                                                            void *neighbor_task_funcptr,
                                                            void *neighbor_args)
{
    acemesh_task_generator_with_neighbors(task_funcptr, args, args_size, cxx_this_pointer,
                                          neighbor_task_funcptr, neighbor_args);
}

extern "C" void acemesh_spawn_and_wait_(int print_graph) { acemesh_spawn_and_wait(print_graph); }
#ifdef REUSE_GRAPH
extern "C" int acemesh_dag_start_(int dagNo) { return acemesh_dag_start_vec(dagNo, 0, 0, 0, 0); }
#endif
extern "C" void acemesh_push_neighbor_addr_(void *addr) { acemesh_push_neighbor_addr(addr); }

extern "C" void acemesh_mpi_rank_(int rank) { aceMesh_MPI_rank(rank); }



#ifdef MASTER
#ifdef MPI_SURPPORT
extern "C" void acemesh_set_suspend_(int mode) { acemesh_set_suspend(mode); }
#ifdef MTEST_LIGHT
extern "C" void acemesh_set_mtest_handle_(int *h1, int *h2, int kind1, int kind2)
{
    acemesh_set_mtest_handle(h1, h2, kind1, kind2);
}
#endif
#endif
extern "C" void acemesh_task_map_master_() { acemesh_task_map_master(); }
#endif
#ifdef LDMM
extern "C" void acemesh_register_invalidatetab_swf_(void *array_addr, int ndim, int *tile_no)
{
    acemesh_register_invalidatetab(array_addr, ndim, tile_no);
}
extern "C" void ldmbuf_add_swf_(int *tilesize, int dim, int tile_num)
{
    ldmbuf_add(tilesize, dim, tile_num);
}
#endif


#ifdef INIT_OFFSET
#define F_INIT_OFFSET 1
#endif
extern "C" int acemesh_get_loop_tile_num_(int loop_tile_size, int loop_index_lb, int loop_index_ub)
{
#ifdef INIT_OFFSET
    return acemesh_get_loop_tile_num_with_init_offset(loop_tile_size, loop_index_lb, loop_index_ub,F_INIT_OFFSET);
#else
    return acemesh_get_loop_tile_num(loop_tile_size, loop_index_lb, loop_index_ub);
#endif
}
extern "C" int acemesh_get_tile_no_by_index_(int loop_tile_size, int compute_index,
                                             int loop_index_lb)
{
#ifdef INIT_OFFSET
    return acemesh_get_tile_no_by_index_with_init_offset(loop_tile_size, compute_index, loop_index_lb,F_INIT_OFFSET);
#else
    return acemesh_get_tile_no_by_index(loop_tile_size, compute_index, loop_index_lb);
#endif
}
extern "C" int acemesh_get_loop_tile_start_index_by_tile_no_(int loop_tile_size, int loop_tile_no,
                                                             int loop_index_lb)
{
#ifdef INIT_OFFSET
    return acemesh_get_loop_tile_start_index_by_tile_no_with_init_offset(loop_tile_size, loop_tile_no,
                                                        loop_index_lb,F_INIT_OFFSET);
#else
    return acemesh_get_loop_tile_start_index_by_tile_no(loop_tile_size, loop_tile_no,
                                                        loop_index_lb);
#endif
}
extern "C" int acemesh_get_loop_tile_start_index_by_tile_no_with_ub_(int loop_tile_size, int loop_tile_no,                                                              
                                                             int loop_index_lb, int loop_index_ub)
{    return acemesh_get_loop_tile_start_index_by_tile_no_with_ub(loop_tile_size, loop_tile_no,
                                                        loop_index_lb, loop_index_ub);
}


extern "C" int acemesh_get_loop_tile_end_index_by_tile_no_(int loop_tile_size, int loop_tile_no,
                                                           int loop_index_lb, int loop_index_ub)
{
#ifdef INIT_OFFSET
    return acemesh_get_loop_tile_end_index_by_tile_no_with_init_offset(loop_tile_size, loop_tile_no, loop_index_lb,
                                                      loop_index_ub,F_INIT_OFFSET);
#else
    return acemesh_get_loop_tile_end_index_by_tile_no(loop_tile_size, loop_tile_no, loop_index_lb,
                                                      loop_index_ub);
#endif
}
extern "C" int acemesh_get_data_tile_start_index_by_index_(int tile_size, int compute_index,
                                                           int array_dim_lb)
{
#ifdef INIT_OFFSET
    return acemesh_get_data_tile_start_index_by_index_with_init_offset(tile_size, compute_index, array_dim_lb,F_INIT_OFFSET);
#else
    return acemesh_get_data_tile_start_index_by_index(tile_size, compute_index, array_dim_lb);
#endif
}

extern "C" int acemesh_get_loop_tile_num_with_init_offset_(int loop_tile_size, int loop_index_lb,
                                                           int loop_index_ub, int init_offset)
{
    return acemesh_get_loop_tile_num_with_init_offset(loop_tile_size, loop_index_lb, loop_index_ub,
                                                      init_offset);
}
extern "C" int acemesh_get_loop_tile_start_index_by_tile_no_with_init_offset_(int loop_tile_size,
                                                                              int loop_tile_no,
                                                                              int loop_index_lb,
                                                                              int init_offset)
{
    return acemesh_get_loop_tile_start_index_by_tile_no_with_init_offset(
        loop_tile_size, loop_tile_no, loop_index_lb, init_offset);
}
extern "C" int acemesh_get_loop_tile_end_index_by_tile_no_with_init_offset_(
    int loop_tile_size, int loop_tile_no, int loop_index_lb, int loop_index_ub, int init_offset)
{
    return acemesh_get_loop_tile_end_index_by_tile_no_with_init_offset(
        loop_tile_size, loop_tile_no, loop_index_lb, loop_index_ub, init_offset);
}
extern "C" int acemesh_get_data_tile_start_index_by_index_with_init_offset_(int tile_size,
                                                                            int compute_index,
                                                                            int array_dim_lb,
                                                                            int init_offset)
{
    return acemesh_get_data_tile_start_index_by_index_with_init_offset(tile_size, compute_index,
                                                                       array_dim_lb, init_offset);
}
extern "C" int acemesh_get_tile_no_by_index_with_init_offset_(int tile_size, int index, int lb,
                                                              int init_offset)
{
    return acemesh_get_tile_no_by_index_with_init_offset(tile_size, index, lb, init_offset);
}
#endif
